# -*- coding: utf-8 -*-
import scrapy

import re
import logging
import urllib.parse
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class GuiyangSpider(scrapy.Spider):
    name = 'guiyang'
    allowed_domains = ['gyfc.net.cn']
    start_urls = ['https://www.gyfc.net.cn/2_proInfo/index.aspx']
    room_li_temp_url = 'https://www.gyfc.net.cn/pro_query/index/floorView.aspx?dongID={}&danyuan={}&qu={}&yszh={}'  # 房号列表url
    custom_settings = {
        'DOWNLOADER_MIDDLEWARES_BASE': {
            'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
            'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
            'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
            'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': None,
            'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
            'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
            'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
            'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
            'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 800,
            'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
            'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
        },
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
            "FDC_spider.middlewares.CustomRetryMiddleware": 500,
            'FDC_spider.middlewares.UserAgentMiddleware': 544,
            'FDC_spider.middlewares.GuiyangMiddleware': 545,  # http协议升级为https协议
            # 'FDC_spider.middlewares.InnerNetProxyMiddleware': 545,
            # 'FDC_spider.middlewares.OuterNetProxyMiddleware': 545,  # 启用redis中的代理，需要同时开启 RedisConnPipeline
            'FDC_spider.middlewares.KuaidailiSfpmProxyMiddleware': 546,  # 快代理
        },
        'ITEM_PIPELINES': {
            # 'FDC_spider.pipelines.RedisConnPipeline': 299,  # 启用redis中的代理，需要同时开启 OuterNetProxyMiddleware
            'FDC_spider.pipelines.FdcSpiderPipeline': 300,
            'FDC_spider.pipelines.GuiyangPipeline': 327,
            # 'FDC_spider.pipelines.GuiyangCsvPipeline': 377,
            # 'FDC_spider.pipelines.MongoClientPipeline': 350,  # 将数据保存到mongo
        },
    }

    def parse(self, response):
        current_page = response.xpath("//div[@id='ProInfo1_AspNetPager1']//font[3]//text()").extract_first()
        total_page = response.xpath("//div[@id='ProInfo1_AspNetPager1']//font[2]//text()").extract_first()
        # 获取楼盘列表
        div_li = response.xpath("//div[@id='right']/table/tr/td/div[position()!=last()]")
        if len(div_li):
            for div in div_li:
                item_eg = FdcEstateGuidItem()
                name_str = div.xpath("./table/tr/td/table//tr[1]/td[3]/text()").extract_first()
                districtName_li = re.findall(r'【(.*)】', name_str)
                estateName_li = re.findall(r'】(.*)', name_str)
                item_eg['districtName'] = districtName_li[0] if len(districtName_li) and districtName_li[
                    0] != 'NULL' else None
                item_eg['estateName'] = estateName_li[0] if len(estateName_li) else None
                estateAddress_str = div.xpath("./table/tr/td/table//tr[2]/td[2]/text()").extract_first()
                item_eg['estateAddress'] = re.sub(r"\s|'\xa0'|[[]", '',
                                                  estateAddress_str) if estateAddress_str else None
                # 许可证列表url
                permit_li_url = div.xpath("./table/tr/td[3]//a/@href").extract_first()
                yield scrapy.Request(
                    permit_li_url,
                    callback=self.parse_permit_li,
                    meta={'item_eg': deepcopy(item_eg)},
                    priority=2,
                )
        else:
            logger.error('{}  {}/{} 楼盘列表获取为空'.format(response.request.url, current_page, total_page))

        # 翻页
        print('>>>>>>>>第{}/{}页<<<<<<<<'.format(current_page, total_page))
        next_page_url = response.xpath("//a[text()='下一页']/@href").extract_first()
        if next_page_url:
            yield response.follow(
                next_page_url,
                priority=1,
            )

    def parse_permit_li(self, response):
        """
        获取许可证列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取许可证列表
        div_li = response.xpath("//div[@id='proInfodetail_panResult']/table/tr/td/div[not(@id)]")
        if len(div_li):
            for div in div_li:
                item_eg['preSalePermit'] = div.xpath("./table/tr/td[1]/table/tr[1]/td[3]/a/text()").extract_first()
                item_eg['projectName'] = div.xpath("./table/tr/td[1]/table/tr[2]/td[2]/text()").extract_first()
                item_eg['estateUrl'] = div.xpath("./table/tr/td[1]/table/tr[1]/td[3]/a/@href").extract_first()
                yield scrapy.Request(
                    item_eg['estateUrl'],
                    callback=self.parse_estate,
                    meta=dict(item_eg=deepcopy(item_eg)),
                    priority=4,
                )
        else:
            logger.warning(
                '{}  {}-{} 预售许可证列表为空'.format(response.request.url, item_eg['districtName'], item_eg['estateName']))

        # 翻页
        next_page_url = response.xpath("//a[text()='下一页']/@href").extract_first()
        if next_page_url:
            yield response.follow(
                next_page_url,
                callback=self.parse_permit_li,
                meta=dict(item_eg=deepcopy(item_eg)),
                priority=3,
            )

    def parse_estate(self, response):
        """
        获取楼盘详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        projectNature = response.xpath("//table[@id='Table2']/tr[5]/td[2]/text()").extract_first()
        developerName = response.xpath("//table[@id='Table2']/tr[6]/td[2]/text()").extract_first()
        salesTel = response.xpath("//table[@id='Table2']/tr[7]/td[2]/text()").extract_first()
        item_eg['projectNature'], item_eg['developerName'], item_eg['salesTel'] = (i.strip() if i else None for i in
                                                                                   [projectNature, developerName,
                                                                                    salesTel])
        item_eg['estateTotalArea'] = response.xpath(
            "//td[contains(text(),'建筑面积')]/following-sibling::td[1]/span/text()").extract_first()
        item_eg['coverageArea'] = response.xpath(
            "//td[contains(text(),'占地面积')]/following-sibling::td[1]/span/text()").extract_first()
        yield item_eg

        # 楼栋列表url
        buildingUrl = response.xpath("//img[@alt='楼盘栋信息']/../@href").extract_first()
        yield response.follow(
            buildingUrl,
            callback=self.parse_building_li,
            meta=dict(item_eg=deepcopy(item_eg)),
            priority=5,
        )

    def parse_building_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取楼栋列表
        tr_li = response.xpath("//table[@id='ContentPlaceHolder1_dong_info1_dg1']//tr[position()>1]")
        if len(tr_li):
            for tr in tr_li:
                item_bd = FdcBuildingItem()
                item_bd['districtName'] = item_eg['districtName']
                item_bd['estateName'] = item_eg['estateName']
                item_bd['projectName'] = item_eg['projectName']
                item_bd['preSalePermit'] = item_eg['preSalePermit']
                item_bd['buildingUrl'] = response.request.url
                item_bd['buildingId'] = tr.xpath("./td[1]/text()").extract_first()
                item_bd['blockName'] = tr.xpath("./td[2]/text()").extract_first()
                item_bd['totalRoomNo'] = tr.xpath("./td[3]/text()").extract_first()
                item_bd['saleableRoomNo'] = tr.xpath("./td[4]/text()").extract_first()
                item_bd['preSaleRoomNo'] = tr.xpath("./td[5]/text()").extract_first()
                item_bd['reserveRoomNo'] = tr.xpath("./td[6]/text()").extract_first()
                yield item_bd

                # 单元url(option信息)
                unit_option_url = tr.xpath(".//a/@href").extract_first()
                yield response.follow(
                    unit_option_url,
                    callback=self.parse_unit_option,
                    meta=dict(item_bd=deepcopy(item_bd)),
                    dont_filter=True,
                    priority=7,
                )
        else:
            logger.warning(
                '{}  {}-{}-{} 楼栋列表获取为空'.format(response.request.url, item_eg['districtName'], item_eg['projectName'],
                                               item_eg['preSalePermit']))

        # 翻页
        next_page_url = response.xpath("//a[text()='下一页']/@href").extract_first()
        if next_page_url:
            yield response.follow(
                next_page_url,
                callback=self.parse_building_li,
                meta=dict(item_eg=deepcopy(item_eg)),
                priority=6,
            )

    def parse_unit_option(self, response):
        """
        获取单元option选项、参数信息
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        VIEWSTATE = response.xpath("//input[@id='__VIEWSTATE']/@value").extract_first()
        VIEWSTATEGENERATOR = response.xpath("//input[@id='__VIEWSTATEGENERATOR']/@value").extract_first()
        EVENTVALIDATION = response.xpath("//input[@id='__EVENTVALIDATION']/@value").extract_first()
        # 构造POST请求参数
        data = {
            '__EVENTTARGET': '_ctl0$ContentPlaceHolder1$floor_list1$ddlDong',
            '__VIEWSTATE': VIEWSTATE,
            '__VIEWSTATEGENERATOR': VIEWSTATEGENERATOR,
            '__EVENTVALIDATION': EVENTVALIDATION,
            '_ctl0:ContentPlaceHolder1:floor_list1:ddlDong': str(item_bd['buildingId']),
            '_ctl0:ContentPlaceHolder1:floor_list1:ddlDanyuan': '全部'.encode('gb2312'),
        }
        yield scrapy.FormRequest(
            response.request.url,
            formdata=data,
            callback=self.parse_unit_li,
            meta=dict(item_bd=deepcopy(item_bd)),
            priority=8,
        )

    def parse_unit_li(self, response):
        """
        获取单元列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        room_li_url = response.xpath("//div[@id='Divcontent']//iframe/@src").extract_first()
        district_str = re.findall(r'qu=(.*)&yszh', room_li_url)[0]
        # 获取单元列表
        option_li = response.xpath("//select[@id='ContentPlaceHolder1_floor_list1_ddlDanyuan']/option")
        if len(option_li):
            for option in option_li:
                unit_str = option.xpath("./@value").extract_first()
                if unit_str != '全部':
                    item_rm = FdcRoomItem()
                    item_rm['districtName'] = item_bd['districtName']
                    item_rm['estateName'] = item_bd['estateName']
                    item_rm['projectName'] = item_bd['projectName']
                    item_rm['preSalePermit'] = item_bd['preSalePermit']
                    item_rm['buildingId'] = item_bd['buildingId']
                    item_rm['blockName'] = item_bd['blockName']
                    unitName_str = option.xpath("./text()").extract_first()
                    item_rm['unitName'] = unitName_str if unitName_str != '无单元' else None
                    yield scrapy.Request(
                        GuiyangSpider.room_li_temp_url.format(item_rm['buildingId'],
                                                              urllib.parse.quote(unit_str,
                                                                                 encoding='gb2312') if unit_str else '',
                                                              urllib.parse.quote(district_str, encoding='gb2312'),
                                                              item_rm['preSalePermit']),
                        callback=self.parse_room_li,
                        meta=dict(item_rm=deepcopy(item_rm)),
                        priority=9,
                    )

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        # 获取楼层分组
        div_li = response.xpath("//div[contains(@style,'LEFT:5px')][position()>1]")
        if len(div_li):
            for i, div in enumerate(div_li):
                item_rm['roomFloor'] = div.xpath(".//text()").extract_first()
                if i < len(div_li) - 1:
                    item_total_count = len(div.xpath("./following-sibling::div[not(contains(@style,'LEFT:5px'))]"))
                    next_item_total_count = len(response.xpath(
                        "//div[contains(@style,'LEFT:5px')][position()={}]/following-sibling::div[not(contains(@style,'LEFT:5px'))]".format(
                            i + 3)))
                    count = item_total_count - next_item_total_count
                    # 获取当前楼层的房号分组
                    rm_div_li = div.xpath("./following-sibling::div[position()<={}]".format(count))
                else:
                    # 获取当前楼层的房号分组
                    rm_div_li = div.xpath("./following-sibling::div[not(contains(@style,'LEFT:5px'))]")
                if len(rm_div_li):
                    for rm_div in rm_div_li:
                        title_str = rm_div.xpath("./@title").extract_first()
                        if title_str:
                            item_rm['roomNo'] = rm_div.xpath(".//text()").extract_first()
                            title_li = re.sub(r'\s', ';', title_str).split(';')
                            try:
                                assert (len(title_li) == 2 or len(title_li) >= 4)
                                if len(title_li) >= 4:
                                    item_rm['saleStatus'] = title_li[0]
                                    item_rm['roomArea'] = title_li[1]
                                    item_rm['roomUse'] = title_li[2]
                                    item_rm['roomStructure'] = title_li[4] if len(title_li) == 5 else None
                                if len(title_li) == 2:
                                    item_rm['saleStatus'] = title_li[0]
                                    item_rm['roomArea'] = title_li[1]
                                    item_rm['roomUse'] = None
                                    item_rm['roomStructure'] = None
                            except Exception as e:
                                logger.warning('{} {}-{}-{}-{} 房号详情提出错误  error:{}'.format(response.request.url,
                                                                                          item_rm['projectName'],
                                                                                          item_rm['blockName'],
                                                                                          item_rm['unitName'],
                                                                                          item_rm['roomFloor'], e))
                            else:
                                yield item_rm
                else:
                    logger.warning(
                        '{}  {}-{}-{}-{}-{}-{}-{} 该楼层房号列表获取为空'.format(response.request.url, item_rm['districtName'],
                                                                      item_rm['estateName'],
                                                                      item_rm['projectName'], item_rm['preSalePermit'],
                                                                      item_rm['blockName'], item_rm['unitName'],
                                                                      item_rm['roomFloor']))
        else:
            if item_rm['unitName']:
                logger.warning(
                    '{}  {}-{}-{}-{}-{}-{} 楼层列表获取为空'.format(response.request.url, item_rm['districtName'],
                                                            item_rm['estateName'],
                                                            item_rm['projectName'], item_rm['preSalePermit'],
                                                            item_rm['blockName'], item_rm['unitName']))
